package cn.spark.study.core;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.storage.StorageLevel;

/**
 * RDD的持久化
 * cache()或者persist()的使用，是有规则的
 * 必须在transformation或者textFile等创建了一个RDD之后，直接连续调用cache()或persist()才可以
 * 如果你先创建一个RDD，然后单独另起一行执行cache()或persist()方法，是没有用的
 * 而且，会报错，大量的文件会丢失
 *
 * @author jun.zhang6
 * @date 2020/11/6
 */
public class Persist {
    public static void main(String[] args) {
        SparkConf sparkConf = new SparkConf()
                .setAppName("Persist")
                .setMaster("local");

        JavaSparkContext sc = new JavaSparkContext(sparkConf);

        JavaRDD<String> linesRdd = sc.textFile("C:\\Users\\jun.zhang6\\Desktop\\spark.txt")
                //.cache();
                .persist(StorageLevel.MEMORY_ONLY());

        long beginTime = System.currentTimeMillis();
        long count = linesRdd.count();
        System.out.println(count);
        long endTime = System.currentTimeMillis();
        System.out.println("cost " + (endTime - beginTime) + " milliseconds.");

        beginTime = System.currentTimeMillis();

        count = linesRdd.count();
        System.out.println(count);

        endTime = System.currentTimeMillis();
        System.out.println("cost " + (endTime - beginTime) + " milliseconds.");

        sc.close();
    }
}
